{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Word Embedding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Looking in indexes: http://mirrors.tencentyun.com/pypi/simple\n",
      "Collecting gensim\n",
      "  Downloading http://mirrors.tencentyun.com/pypi/packages/2b/e0/fa6326251692056dc880a64eb22117e03269906ba55a6864864d24ec8b4e/gensim-3.8.3-cp36-cp36m-manylinux1_x86_64.whl (24.2 MB)\n",
      "\u001b[K     |████████████████████████████████| 24.2 MB 11.6 MB/s eta 0:00:01\n",
      "\u001b[?25hCollecting smart-open>=1.8.1\n",
      "  Downloading http://mirrors.tencentyun.com/pypi/packages/0b/8e/464b06f5efd26f2dc16ce7bd1662c2f31cadf9104fdbcbf5994674cc3a51/smart_open-2.1.0.tar.gz (116 kB)\n",
      "\u001b[K     |████████████████████████████████| 116 kB 74.9 MB/s eta 0:00:01\n",
      "\u001b[?25hRequirement already satisfied: numpy>=1.11.3 in /opt/conda/envs/pytorch_py3/lib/python3.6/site-packages (from gensim) (1.18.5)\n",
      "Requirement already satisfied: scipy>=0.18.1 in /opt/conda/envs/pytorch_py3/lib/python3.6/site-packages (from gensim) (1.5.0)\n",
      "Requirement already satisfied: six>=1.5.0 in /opt/conda/envs/pytorch_py3/lib/python3.6/site-packages (from gensim) (1.15.0)\n",
      "Requirement already satisfied: requests in /opt/conda/envs/pytorch_py3/lib/python3.6/site-packages (from smart-open>=1.8.1->gensim) (2.24.0)\n",
      "Collecting boto\n",
      "  Downloading http://mirrors.tencentyun.com/pypi/packages/23/10/c0b78c27298029e4454a472a1919bde20cb182dab1662cec7f2ca1dcc523/boto-2.49.0-py2.py3-none-any.whl (1.4 MB)\n",
      "\u001b[K     |████████████████████████████████| 1.4 MB 17.7 MB/s eta 0:00:01\n",
      "\u001b[?25hCollecting boto3\n",
      "  Downloading http://mirrors.tencentyun.com/pypi/packages/c9/a5/e06492d12da34135728559aa18ba6bc841a82cea5a5b3bcbb643ea2dbe07/boto3-1.14.19-py2.py3-none-any.whl (128 kB)\n",
      "\u001b[K     |████████████████████████████████| 128 kB 16.6 MB/s eta 0:00:01\n",
      "\u001b[?25hRequirement already satisfied: idna<3,>=2.5 in /opt/conda/envs/pytorch_py3/lib/python3.6/site-packages (from requests->smart-open>=1.8.1->gensim) (2.10)\n",
      "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /opt/conda/envs/pytorch_py3/lib/python3.6/site-packages (from requests->smart-open>=1.8.1->gensim) (1.25.9)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /opt/conda/envs/pytorch_py3/lib/python3.6/site-packages (from requests->smart-open>=1.8.1->gensim) (2020.6.20)\n",
      "Requirement already satisfied: chardet<4,>=3.0.2 in /opt/conda/envs/pytorch_py3/lib/python3.6/site-packages (from requests->smart-open>=1.8.1->gensim) (3.0.4)\n",
      "Collecting botocore<1.18.0,>=1.17.19\n",
      "  Downloading http://mirrors.tencentyun.com/pypi/packages/e4/33/60eb84ff5dfb26a0c3ae606ba42686f23e9fd5038afddd5882d37623d589/botocore-1.17.19-py2.py3-none-any.whl (6.3 MB)\n",
      "\u001b[K     |████████████████████████████████| 6.3 MB 11.2 MB/s eta 0:00:01\n",
      "\u001b[?25hCollecting jmespath<1.0.0,>=0.7.1\n",
      "  Downloading http://mirrors.tencentyun.com/pypi/packages/07/cb/5f001272b6faeb23c1c9e0acc04d48eaaf5c862c17709d20e3469c6e0139/jmespath-0.10.0-py2.py3-none-any.whl (24 kB)\n",
      "Collecting s3transfer<0.4.0,>=0.3.0\n",
      "  Downloading http://mirrors.tencentyun.com/pypi/packages/69/79/e6afb3d8b0b4e96cefbdc690f741d7dd24547ff1f94240c997a26fa908d3/s3transfer-0.3.3-py2.py3-none-any.whl (69 kB)\n",
      "\u001b[K     |████████████████████████████████| 69 kB 89.5 MB/s eta 0:00:01\n",
      "\u001b[?25hRequirement already satisfied: python-dateutil<3.0.0,>=2.1 in /opt/conda/envs/pytorch_py3/lib/python3.6/site-packages (from botocore<1.18.0,>=1.17.19->boto3->smart-open>=1.8.1->gensim) (2.8.1)\n",
      "Collecting docutils<0.16,>=0.10\n",
      "  Downloading http://mirrors.tencentyun.com/pypi/packages/22/cd/a6aa959dca619918ccb55023b4cb151949c64d4d5d55b3f4ffd7eee0c6e8/docutils-0.15.2-py3-none-any.whl (547 kB)\n",
      "\u001b[K     |████████████████████████████████| 547 kB 79.1 MB/s eta 0:00:01\n",
      "\u001b[?25hBuilding wheels for collected packages: smart-open\n",
      "  Building wheel for smart-open (setup.py) ... \u001b[?25ldone\n",
      "\u001b[?25h  Created wheel for smart-open: filename=smart_open-2.1.0-py3-none-any.whl size=110317 sha256=52eecfb9c50c9759451344ec66e9392c6024a3c0a13a3d1ccd72935a7b64415a\n",
      "  Stored in directory: /home/tione/.cache/pip/wheels/af/b2/aa/49b1424dd5099959003661d365fe8dcec7c3ddf2290a91a568\n",
      "Successfully built smart-open\n",
      "Installing collected packages: boto, jmespath, docutils, botocore, s3transfer, boto3, smart-open, gensim\n",
      "Successfully installed boto-2.49.0 boto3-1.14.19 botocore-1.17.19 docutils-0.15.2 gensim-3.8.3 jmespath-0.10.0 s3transfer-0.3.3 smart-open-2.1.0\n"
     ]
    }
   ],
   "source": [
    "!pip install gensim"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-05-30T14:45:20.894711Z",
     "start_time": "2020-05-30T14:45:19.925317Z"
    }
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import random\n",
    "import time\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')\n",
    "import gc\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from tqdm import tqdm\n",
    "import gensim\n",
    "from gensim.models.callbacks import CallbackAny2Vec\n",
    "np.random.seed(2020)\n",
    "os.environ['PYTHONHASHSEED'] = '0'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-05-30T14:45:20.903054Z",
     "start_time": "2020-05-30T14:45:20.897237Z"
    }
   },
   "outputs": [],
   "source": [
    "save_path_word2vec = './embedding/word2vec'\n",
    "save_path_glove    = './embedding/glove'\n",
    "save_path_fasttext = './embedding/fasttext'\n",
    "for path in [save_path_word2vec, save_path_glove, save_path_fasttext]:\n",
    "    if not os.path.exists(path):\n",
    "        os.makedirs(path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import logging\n",
    "logging.basicConfig(filename='./embedding/word2vec/train.log', format='%(asctime)s:%(message)s', level=logging.CRITICAL)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-05-30T14:45:37.807525Z",
     "start_time": "2020-05-30T14:45:20.907573Z"
    }
   },
   "outputs": [],
   "source": [
    "df = pd.read_pickle('./processed_data/processed_data_numerical.pkl')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Word2Vec"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-05-30T14:45:37.842644Z",
     "start_time": "2020-05-30T14:45:37.826792Z"
    }
   },
   "outputs": [],
   "source": [
    "class EpochLogger(CallbackAny2Vec):\n",
    "    def __init__(self, name, path):\n",
    "        self.path = path\n",
    "        self.epoch = 0\n",
    "        self.best_loss = None\n",
    "        self.name = name\n",
    "\n",
    "    def on_epoch_end(self, model):\n",
    "        cur_loss = float(model.get_latest_training_loss())\n",
    "#         if self.best_loss is None or cur_loss <= self.best_loss:\n",
    "#             self.best_loss = cur_loss\n",
    "#             model.wv.save_word2vec_format(self.path)\n",
    "        message = \"[{}] Epoch #{} {:.2f}\".format(self.name, self.epoch, cur_loss)\n",
    "        print(message)\n",
    "        logging.critical(message)\n",
    "        model.running_training_loss = 0.0  # word2vec默认是累计损失，会溢出\n",
    "        self.epoch += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "start_time": "2020-05-30T14:45:20.852Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[creative_id] Epoch #0 134217728.00\n",
      "[creative_id] Epoch #1 134217728.00\n",
      "[creative_id] Epoch #2 134217728.00\n",
      "[creative_id] Epoch #3 134217728.00\n",
      "[creative_id] Epoch #4 134217728.00\n",
      "[creative_id] Epoch #5 134217728.00\n",
      "[creative_id] Epoch #6 134217728.00\n",
      "[creative_id] Epoch #7 134217728.00\n"
     ]
    }
   ],
   "source": [
    "for name, epochs in zip(['creative_id', 'ad_id', 'product_id', 'product_category', 'advertiser_id', 'industry'], [80, 80, 20, 20, 20, 20]):\n",
    "    path = os.path.join(save_path_word2vec, '{}_word2vec_sg1_hs0_win20_mc1_size300.txt'.format(name))\n",
    "    input_docs = list(df[name].apply(lambda x: list(x.astype(str))))\n",
    "    w2v = gensim.models.Word2Vec(input_docs, size=300, sg=1, hs=0, alpha=0.025, min_alpha=0, window=20, seed=2020, workers=32, min_count=1, iter=epochs, compute_loss=True, callbacks=[EpochLogger(name, path)])\n",
    "    w2v.wv.save_word2vec_format(path)\n",
    "    del input_docs, w2v\n",
    "    gc.collect()\n",
    "    \n",
    "    \n",
    "embedding_path = './embedding/word2vec'\n",
    "creative_w2v = get_word_embedding(embed_path=os.path.join(embedding_path, 'creative_id_word2vec_sg1_hs0_win20_mc1_size300.txt'), vocab_size=4445721, glove=False)\n",
    "\n",
    "ad_w2v = get_word_embedding(embed_path=os.path.join(embedding_path, 'ad_id_word2vec_sg1_hs0_win20_mc1_size300.txt'), vocab_size=3812203, glove=False)\n",
    "\n",
    "advertiser_w2v = get_word_embedding(embed_path=os.path.join(embedding_path, 'advertiser_id_word2vec_sg1_hs0_win20_mc1_size300.txt'), vocab_size=62966, glove=False)\n",
    "\n",
    "product_w2v = get_word_embedding(embed_path=os.path.join(embedding_path, 'product_id_word2vec_sg1_hs0_win20_mc1_size300.txt'), vocab_size=44316, glove=False)\n",
    "\n",
    "industry_w2v = get_word_embedding(embed_path=os.path.join(embedding_path, 'industry_word2vec_sg1_hs0_win20_mc1_size300.txt'), vocab_size=337, glove=False)\n",
    "\n",
    "product_cate_w2v = get_word_embedding(embed_path=os.path.join(embedding_path, 'product_category_word2vec_sg1_hs0_win20_mc1_size300.txt'), vocab_size=19, glove=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['product_category_word2vec_sg1_hs0_win100_mc1_size128.txt',\n",
       " 'creative_id_word2vec_sg1_hs0_win20_mc1_size300.txt',\n",
       " 'product_category_word2vec_sg1_hs0_win20_mc1_size300.txt',\n",
       " 'creative_id_word2vec_sg1_hs0_win100_mc1_size128.txt',\n",
       " 'advertiser_id_word2vec_sg1_hs0_win20_mc1_size300.txt',\n",
       " 'train.log',\n",
       " 'product_id_word2vec_sg1_hs0_win20_mc1_size300.txt',\n",
       " '.ipynb_checkpoints',\n",
       " 'ad_id_word2vec_sg1_hs0_win10_mc1_size512.txt',\n",
       " 'product_category_word2vec_sg1_hs0_win10_mc1_size512.txt',\n",
       " 'ad_id_word2vec_sg1_hs0_win100_mc1_size128.txt',\n",
       " 'advertiser_id_word2vec_sg1_hs0_win10_mc1_size512.txt',\n",
       " 'product_id_word2vec_sg1_hs0_win100_mc1_size128.txt',\n",
       " 'embedding_w2v_sg1_hs0_win100_size128.npz',\n",
       " 'embedding_w2v_sg1_hs0_win10_size512.npz',\n",
       " 'creative_id_word2vec_sg1_hs0_win10_mc1_size512.txt',\n",
       " 'industry_word2vec_sg1_hs0_win100_mc1_size128.txt',\n",
       " 'industry_word2vec_sg1_hs0_win20_mc1_size300.txt',\n",
       " 'advertiser_id_word2vec_sg1_hs0_win100_mc1_size128.txt',\n",
       " 'industry_word2vec_sg1_hs0_win10_mc1_size512.txt',\n",
       " 'ad_id_word2vec_sg1_hs0_win20_mc1_size300.txt',\n",
       " 'product_id_word2vec_sg1_hs0_win10_mc1_size512.txt']"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "os.listdir('./embedding/word2vec')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_word_embedding(embed_path, vocab_size, glove=False):\n",
    "    pre_embedding = {}\n",
    "    # 用python的生成器读取大文件，并且选取在index中的读入，减少内存消耗\n",
    "    with open(embed_path, encoding='utf8') as f:\n",
    "        first_line = next(f)\n",
    "        word_num, embed_size = int(first_line.split()[0]), int(first_line.split()[1])\n",
    "        if glove:\n",
    "            # glove 是context vector 和 bias vector 的concat\n",
    "            word_num -= 1\n",
    "            embed_size = 2*embed_size\n",
    "        embedding_matrix = np.zeros((vocab_size, embed_size))\n",
    "        for line in tqdm(f, total=word_num):\n",
    "            tmp = line.strip().split() \n",
    "            if tmp[0] == '<unk>':\n",
    "                continue\n",
    "            embedding_matrix[int(tmp[0]), :] = np.array(tmp[1:embed_size+1]).astype(np.float)\n",
    "    return embedding_matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 4445720/4445720 [15:53<00:00, 4662.31it/s]\n",
      "100%|██████████| 3812202/3812202 [13:42<00:00, 4636.61it/s]\n",
      "100%|██████████| 62965/62965 [00:13<00:00, 4691.80it/s]\n",
      "100%|██████████| 44315/44315 [00:09<00:00, 4674.42it/s]\n",
      "100%|██████████| 336/336 [00:00<00:00, 4759.77it/s]\n",
      "100%|██████████| 18/18 [00:00<00:00, 4094.45it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(4445721, 300)\n",
      "(3812203, 300)\n",
      "(62966, 300)\n",
      "(44316, 300)\n",
      "(337, 300)\n",
      "(19, 300)\n"
     ]
    }
   ],
   "source": [
    "# embedding_path = './embedding/word2vec'\n",
    "# creative_w2v = get_word_embedding(embed_path=os.path.join(embedding_path, 'creative_id_word2vec_sg1_hs0_win10_mc1_size300.txt'), vocab_size=4445721, glove=False)\n",
    "\n",
    "# ad_w2v = get_word_embedding(embed_path=os.path.join(embedding_path, 'ad_id_word2vec_sg1_hs0_win10_mc1_size300.txt'), vocab_size=3812203, glove=False)\n",
    "\n",
    "# advertiser_w2v = get_word_embedding(embed_path=os.path.join(embedding_path, 'advertiser_id_word2vec_sg1_hs0_win10_mc1_size300.txt'), vocab_size=62966, glove=False)\n",
    "\n",
    "# product_w2v = get_word_embedding(embed_path=os.path.join(embedding_path, 'product_id_word2vec_sg1_hs0_win10_mc1_size300.txt'), vocab_size=44316, glove=False)\n",
    "\n",
    "# industry_w2v = get_word_embedding(embed_path=os.path.join(embedding_path, 'industry_word2vec_sg1_hs0_win10_mc1_size300.txt'), vocab_size=337, glove=False)\n",
    "\n",
    "# product_cate_w2v = get_word_embedding(embed_path=os.path.join(embedding_path, 'product_category_word2vec_sg1_hs0_win10_mc1_size300.txt'), vocab_size=19, glove=False)\n",
    "# print(creative_w2v.shape)\n",
    "# print(ad_w2v.shape)\n",
    "# print(advertiser_w2v.shape)\n",
    "# print(product_w2v.shape)\n",
    "# print(industry_w2v.shape)\n",
    "# print(product_cate_w2v.shape)\n",
    "\n",
    "# # 保存好embedding，便于下次直接读取\n",
    "# np.savez(os.path.join(embedding_path, 'embedding_w2v_sg1_hs0_win10_size300'), creative_w2v=creative_w2v.astype(np.float16), ad_w2v=ad_w2v.astype(np.float16), advertiser_w2v=advertiser_w2v.astype(np.float16), product_w2v=product_w2v.astype(np.float16), industry_w2v=industry_w2v.astype(np.float16), product_cate_w2v=product_cate_w2v.astype(np.float16))\n",
    "\n",
    "\n",
    "# embedding_path = './embedding/word2vec'\n",
    "# creative_w2v = get_word_embedding(embed_path=os.path.join(embedding_path, 'creative_id_word2vec_sg1_hs0_win10_mc1_size128.txt'), vocab_size=4445721, glove=False)\n",
    "\n",
    "# ad_w2v = get_word_embedding(embed_path=os.path.join(embedding_path, 'ad_id_word2vec_sg1_hs0_win10_mc1_size128.txt'), vocab_size=3812203, glove=False)\n",
    "\n",
    "# advertiser_w2v = get_word_embedding(embed_path=os.path.join(embedding_path, 'advertiser_id_word2vec_sg1_hs0_win10_mc1_size128.txt'), vocab_size=62966, glove=False)\n",
    "\n",
    "# product_w2v = get_word_embedding(embed_path=os.path.join(embedding_path, 'product_id_word2vec_sg1_hs0_win10_mc1_size128.txt'), vocab_size=44316, glove=False)\n",
    "\n",
    "# industry_w2v = get_word_embedding(embed_path=os.path.join(embedding_path, 'industry_word2vec_sg1_hs0_win10_mc1_size128.txt'), vocab_size=337, glove=False)\n",
    "\n",
    "# product_cate_w2v = get_word_embedding(embed_path=os.path.join(embedding_path, 'product_category_word2vec_sg1_hs0_win10_mc1_size128.txt'), vocab_size=19, glove=False)\n",
    "# print(creative_w2v.shape)\n",
    "# print(ad_w2v.shape)\n",
    "# print(advertiser_w2v.shape)\n",
    "# print(product_w2v.shape)\n",
    "# print(industry_w2v.shape)\n",
    "# print(product_cate_w2v.shape)\n",
    "\n",
    "# # 保存好embedding，便于下次直接读取\n",
    "# np.savez(os.path.join(embedding_path, 'embedding_w2v_sg1_hs0_win10_size128'), creative_w2v=creative_w2v.astype(np.float16), ad_w2v=ad_w2v.astype(np.float16), advertiser_w2v=advertiser_w2v.astype(np.float16), product_w2v=product_w2v.astype(np.float16), industry_w2v=industry_w2v.astype(np.float16), product_cate_w2v=product_cate_w2v.astype(np.float16))\n",
    "\n",
    "\n",
    "embedding_path = './embedding/word2vec'\n",
    "creative_w2v = get_word_embedding(embed_path=os.path.join(embedding_path, 'creative_id_word2vec_sg1_hs0_win20_mc1_size300.txt'), vocab_size=4445721, glove=False)\n",
    "\n",
    "ad_w2v = get_word_embedding(embed_path=os.path.join(embedding_path, 'ad_id_word2vec_sg1_hs0_win20_mc1_size300.txt'), vocab_size=3812203, glove=False)\n",
    "\n",
    "advertiser_w2v = get_word_embedding(embed_path=os.path.join(embedding_path, 'advertiser_id_word2vec_sg1_hs0_win20_mc1_size300.txt'), vocab_size=62966, glove=False)\n",
    "\n",
    "product_w2v = get_word_embedding(embed_path=os.path.join(embedding_path, 'product_id_word2vec_sg1_hs0_win20_mc1_size300.txt'), vocab_size=44316, glove=False)\n",
    "\n",
    "industry_w2v = get_word_embedding(embed_path=os.path.join(embedding_path, 'industry_word2vec_sg1_hs0_win20_mc1_size300.txt'), vocab_size=337, glove=False)\n",
    "\n",
    "product_cate_w2v = get_word_embedding(embed_path=os.path.join(embedding_path, 'product_category_word2vec_sg1_hs0_win20_mc1_size300.txt'), vocab_size=19, glove=False)\n",
    "print(creative_w2v.shape)\n",
    "print(ad_w2v.shape)\n",
    "print(advertiser_w2v.shape)\n",
    "print(product_w2v.shape)\n",
    "print(industry_w2v.shape)\n",
    "print(product_cate_w2v.shape)\n",
    "\n",
    "# 保存好embedding，便于下次直接读取\n",
    "np.savez(os.path.join(embedding_path, 'embedding_w2v_sg1_hs0_win20_size300'), creative_w2v=creative_w2v.astype(np.float16), ad_w2v=ad_w2v.astype(np.float16), advertiser_w2v=advertiser_w2v.astype(np.float16), product_w2v=product_w2v.astype(np.float16), industry_w2v=industry_w2v.astype(np.float16), product_cate_w2v=product_cate_w2v.astype(np.float16))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
